#@Authar:Litter_hui
#@time:2020/8/14  10:39
#@name:PyCharm
#---------coding:utf-8--------

def main():
    # 一级url:0风景，1美女，2游戏，3动漫，4影视，5明星，6汽车，7人物，8动物，9人物，10美食，11宗教，12背景
    baseurl_lib = ["http://pic.netbian.com/4kdongman/index"]

    getDate(baseurl)

    print("爬取完成")


# 正则表达式标准
# findhref=re.compile(r'<a href=".*?"')
findhref = re.compile(r'/tupian/\d*.html')
findImg = re.compile(r'src="(/uploads/allimg/.*.jpg)"')
finfTitle = re.compile(r'title="(.*)"/>')
# 爬取数据
flag = 0


def getDate(baseurl):
    # 每页URL跳转
    name = []
    link = []
    for i in range(0, 137):
        if i == 0:
            url = baseurl + ".html"
        else:
            url = baseurl + "_" + str(i + 1) + ".html"
        # print(url)

        # 解析数据
        html = askURL(url)
        soup = BeautifulSoup(html, 'html.parser')
        for item in soup.find_all('ul', class_="clearfix"):
            item = str(item)
            # print(item)
            href = re.findall(findhref, item)
            # print(href)
            # print(len(href))

            # 爬取二级网页
            for i in range(0, len(href)):
                lastUrl = "http://pic.netbian.com" + href[i]
                # print(lastUrl)
                lasthtml = askURL(lastUrl)
                # print(lasthtml)
                lastsoup = BeautifulSoup(lasthtml, 'html.parser')
                for src in lastsoup.find_all('div', class_="photo-pic"):
                    data = []
                    src = str(src)
                    # print(src)

                    jpg = "http://pic.netbian.com" + str(re.findall(findImg, src)[0])
                    global flag
                    flag += 1
                    print("第%d张完成" % flag)
                    # print(jpg)
                    obj = urllib.request.urlopen(jpg)
                    obj = obj.read()

                    title = re.findall(finfTitle, src)
                    print(title)
                    saveDate(obj, title[0])

    return name, link


def askURL(url):
    # 模拟浏览器head
    head = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) "
                          "Chrome/78.0.3904.108 Safari/537.36"}
    request = urllib.request.Request(url, headers=head)

    html = ""
    try:
        response = urllib.request.urlopen(request)
        # html = response.read().decode("utf-8")
        html = response.read().decode("gbk")
        # print(html)
    except urllib.error.URLError as e:
        if hasattr(e, 'code'):
            print(e.code)
        if hasattr(e, 'reason'):
            print(e.reason)
    return html


# 保存数据
def saveDate(jpg, title):
    if flag == 1:
        os.mkdir("盗亦有道")
    try:
        f = open(r'盗亦有道\\' + str(title) + '.jpg', 'wb')
        f.write(jpg)
        f.close()
    except OSError:
        pass


if __name__ == '__main__':
    main()
    print("所长在等你")